package com.foo.preprocessing;

import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;

import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import weka.core.Attribute;
import weka.core.FastVector;
import weka.core.Instance;
import weka.core.Instances;

import com.foo.constants.Constants;
import com.foo.preprocessing.DataBean;

/*
 * Class which creates an Arff File from the XML data source.
 * This class contains different Arff files used for the validation of our classifier
 * The class attribute is the category attribute 
 * The different Arff files Created are:
 * 1. Arff File containing the XML tags like Source, Title, Description and Category 
 * 2. Arff File containing the XML tags like Source and Category
 * 3. Arff File containing the XML tags like Title and Category
 * 4. Arff File containing the XML tags like Description and Category
 * 5. Arff File containing the XML tags like Source, Title and Category
 * 6. Arff File containing the XML tags like Title, Description and Category
 * 7. Arff File containing the XML tags like Source, Description and Category 
 */
public class CreatingArffFile 
{
	public static List<DataBean> articles;
	/*
	 * readXMLFile() reads the data from the XML file and returns an object of the class variable.
	 * Missing Values are indicated by "?" string
	 */
	private List<DataBean> readXMLFile()
	{
		File newsData = new File(Constants.FILE_PATH);
				
		try
		{
			DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
			DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
			Document doc = dBuilder.parse(newsData);
			doc.getDocumentElement().normalize();
			NodeList nodes = doc.getElementsByTagName(Constants.ROOT_ATTRIBUTE);
			boolean invalidData = false;		
			for (int i = 0; i < nodes.getLength(); i++) 	
			{
				Node node = nodes.item(i);

				if (node.getNodeType() == Node.ELEMENT_NODE) 
				{
					Element element = (Element) node;
					int datasetCount = element.getElementsByTagName(Constants.SOURCE_ATTRIBUTE).getLength();
					System.out.println("Found " + datasetCount + " elements in XML file");
					articles = new ArrayList<DataBean>(datasetCount);
					
					for (int j=0;j<datasetCount;j++)
					{
						DataBean newdata= new DataBean();
						if (element.getElementsByTagName(Constants.SOURCE_ATTRIBUTE).item(j).getChildNodes().item(0) != null) {
						    newdata.setSource(element.getElementsByTagName(Constants.SOURCE_ATTRIBUTE).item(j).getChildNodes().item(0).getNodeValue());
						}
						else
						{
							newdata.setSource(Constants.MISSING_VALUE);
						}
						if (element.getElementsByTagName(Constants.DESCRIPTION_ATTRIBUTE).item(j).getChildNodes().item(0) != null) 
						{
							if(element.getElementsByTagName(Constants.DESCRIPTION_ATTRIBUTE).item(j).getChildNodes().item(0).getNodeValue()!= null || 
									element.getElementsByTagName(Constants.DESCRIPTION_ATTRIBUTE).item(j).getChildNodes().item(0).getNodeValue()!= "")
							{
								String description = element.getElementsByTagName(Constants.DESCRIPTION_ATTRIBUTE).item(j).getChildNodes().item(0).getNodeValue();
								description = description.replaceAll(Constants.INVALID_STRING_REGULAR_EXPRESSION, "");
								
								newdata.setDescription(description);
							}
							else
							{
								newdata.setDescription(Constants.MISSING_VALUE);
							}														
						}
						else
						{
							newdata.setDescription(Constants.MISSING_VALUE);
						}
						if (element.getElementsByTagName(Constants.TITLE_ATTRIBUTE).item(j).getChildNodes().item(0) != null) 
						{
							if(element.getElementsByTagName(Constants.TITLE_ATTRIBUTE).item(j).getChildNodes().item(0).getNodeValue() != null ||
									element.getElementsByTagName(Constants.TITLE_ATTRIBUTE).item(j).getChildNodes().item(0).getNodeValue() != "")
							{
								newdata.setTitle(
										element.getElementsByTagName(Constants.TITLE_ATTRIBUTE).item(j).getChildNodes().item(0).getNodeValue());	
							}
							else
							{
								newdata.setTitle(Constants.MISSING_VALUE);	
							}
							
						}
						else
						{
							newdata.setTitle(Constants.MISSING_VALUE);
						}
						if (element.getElementsByTagName(Constants.CATEGORY_ATTRIBUTE).item(j).getChildNodes().item(0) != null) 
						{
							if(element.getElementsByTagName(Constants.CATEGORY_ATTRIBUTE).item(j).getChildNodes().item(0).getNodeValue() != null || 
									element.getElementsByTagName(Constants.CATEGORY_ATTRIBUTE).item(j).getChildNodes().item(0).getNodeValue() != "")
							{
								String categoryName = element.getElementsByTagName(
														Constants.CATEGORY_ATTRIBUTE).item(j).getChildNodes().item(0).getNodeValue();
								if(Pattern.matches("[0-9]+", categoryName))
								{
									invalidData = true;
								}
								else
								{
									newdata.setCategory(
								    		element.getElementsByTagName(Constants.CATEGORY_ATTRIBUTE).item(j).getChildNodes().item(0).getNodeValue());
								}
							}
							else
							{
								newdata.setCategory(Constants.MISSING_VALUE);
								invalidData = true;								
							}
						    
						}
						else
						{
							newdata.setCategory(Constants.MISSING_VALUE);
							invalidData = true;
						}
						if(!invalidData)
						{
							articles.add(newdata);	
						}	
						else
						{
							invalidData = false;
						}
					}					
				}
			}	
		}
		catch(Exception e)
		{
			System.out.println("Exception Occured in ReadXML Function:  " + e.getMessage());
		}
		return articles;
	}
	
	/*
	 * create_ArffFile() create an Arff file with attributes like source, title, description and category as Class attribute/ target attribute
	 * Here all attributes created will have the same weights and instances will have same weights of 1.0
	 */
	public Instances create_ArffFile() throws Exception
	{
		FastVector atts = new FastVector(4);
		atts.addElement(new Attribute(Constants.SOURCE_ATTRIBUTE, (FastVector) null));
		atts.addElement(new Attribute(Constants.TITLE_ATTRIBUTE, (FastVector) null));
		atts.addElement(new Attribute(Constants.DESCRIPTION_ATTRIBUTE, (FastVector) null));
		atts.addElement(new Attribute(Constants.CATEGORY_ATTRIBUTE, (FastVector) null));
		
		Instances data = new Instances(Constants.RELATION_NAME, atts, 0);
		
		/*Class Attribute is the attribute to which the instance has to be classified. 
		Here we want all the class instances to be classified to category attribute*/
		data.setClass(data.attribute(Constants.CATEGORY_ATTRIBUTE)); 
		
		try {
			if(articles == null)
			{
				articles = readXMLFile();
			}
			
			System.out.println("number of articles " + articles.size());
			if(articles.size() > 0)
			{
				for(int i=0;i<articles.size();i++)
				{
					double[] newInst = new double[4];
					
					newInst[0] =(double)data.attribute(Constants.SOURCE_ATTRIBUTE).addStringValue(articles.get(i).getSource());
					newInst[1] =(double)data.attribute(Constants.TITLE_ATTRIBUTE).addStringValue(articles.get(i).getTitle());
					newInst[2] =(double)data.attribute(Constants.DESCRIPTION_ATTRIBUTE).addStringValue(articles.get(i).getDescription());
					newInst[3] =(double)data.attribute(Constants.CATEGORY_ATTRIBUTE).addStringValue(articles.get(i).getCategory());
					
					//Assign Equal Weight for all instances
					data.add(new Instance(1.0, newInst));
			}
				
				//System.out.println("Arff Dataset is: " + data);
			}
			
	    } 
		catch (Exception e) {
			System.err.println("failed to create arff file. "+ e.getMessage());
		}
		return data;		
	}
	
	/*
	 * create_ArffFile_With_Weight() create an Arff file with attributes like source, title, description and category as Class attribute/ target attribute
	 * Here each attributes is assigned the user defined weights and instances will have equal weights of 1.0
	 */
	public Instances create_ArffFile_With_Weight(double source_weight, double title_weight, double desc_weight) throws Exception
	{
		
		FastVector atts = new FastVector(4);
		atts.addElement(new Attribute(Constants.SOURCE_ATTRIBUTE, (FastVector) null));
		atts.addElement(new Attribute(Constants.TITLE_ATTRIBUTE, (FastVector) null));
		atts.addElement(new Attribute(Constants.DESCRIPTION_ATTRIBUTE, (FastVector) null));
		atts.addElement(new Attribute(Constants.CATEGORY_ATTRIBUTE, (FastVector) null));
		
		Instances data = new Instances(Constants.RELATION_NAME, atts, 0);
		
		data.attribute(Constants.SOURCE_ATTRIBUTE).setWeight(source_weight);
		data.attribute(Constants.TITLE_ATTRIBUTE).setWeight(title_weight);
		data.attribute(Constants.DESCRIPTION_ATTRIBUTE).setWeight(desc_weight);
		
		/*Class Attribute is the attribute to which the instance has to be classified. 
		Here we want all the class instances to be classified to category attribute*/
		data.setClass(data.attribute(Constants.CATEGORY_ATTRIBUTE)); 
		
		try {
			if(articles == null)
			{
				articles = readXMLFile();
			}
			System.out.println("number of articles " + articles.size());
			if(articles.size() > 0)
			{
				for(int i=0;i<articles.size();i++)
				{
					double[] newInst = new double[4];
					
					newInst[0] =(double)data.attribute(Constants.SOURCE_ATTRIBUTE).addStringValue(articles.get(i).getSource());
					newInst[1] =(double)data.attribute(Constants.TITLE_ATTRIBUTE).addStringValue(articles.get(i).getTitle());
					newInst[2] =(double)data.attribute(Constants.DESCRIPTION_ATTRIBUTE).addStringValue(articles.get(i).getDescription());
					newInst[3] =(double)data.attribute(Constants.CATEGORY_ATTRIBUTE).addStringValue(articles.get(i).getCategory());
					
					//Assign Equal Weight for all instances
					data.add(new Instance(1.0, newInst));
				}
				
				//System.out.println("Arff Dataset is: " + data);
			}
			
	    } 
		catch (Exception e) {
			System.err.println("failed to create arff file. "+ e.getMessage());
		}
		return data;		
	}

	/*
	 * create_ArffFile() create an Arff file with a combination of two attributes and category as Class attribute/ target attribute
	 * Here all attributes created will have the same weights and instances will have same weights of 1.0
	 * The combinations could lead to the following ARFF files
	 * 1. Arff File containing the XML tags like Source, Title and Category
	 * 2. Arff File containing the XML tags like Title, Description and Category
	 * 3. Arff File containing the XML tags like Source, Description and Category
	 */
	public Instances create_ArffFile_With_Weight(
			String attribute1, double attribute1_weight, String attribute2, double attribute2_weight) throws Exception
	{
		FastVector atts = new FastVector(3);
		atts.addElement(new Attribute(attribute1, (FastVector) null));
		atts.addElement(new Attribute(attribute2, (FastVector) null));
		atts.addElement(new Attribute(Constants.CATEGORY_ATTRIBUTE, (FastVector) null));
		
		Instances data = new Instances(Constants.RELATION_NAME, atts, 0);
		
		data.attribute(attribute1).setWeight(attribute1_weight);
		data.attribute(attribute2).setWeight(attribute2_weight);
		
		
		/*Class Attribute is the attribute to which the instance has to be classified. 
		Here we want all the class instances to be classified to category attribute*/
		data.setClass(data.attribute(Constants.CATEGORY_ATTRIBUTE)); 
		
		try {
			if(articles == null)
			{
				articles = readXMLFile();
			}
			System.out.println("number of articles " + articles.size());
			if(articles.size() > 0)
			{
				for(int i=0;i<articles.size();i++)
				{
					double[] newInst = new double[3];
					
					//Check which attribute entered to add the value
					if(attribute1.equals(Constants.SOURCE_ATTRIBUTE))
					{
						newInst[0] =(double)data.attribute(attribute1).addStringValue(articles.get(i).getSource());	
					}
					else if(attribute1.equals(Constants.TITLE_ATTRIBUTE))
					{
						newInst[0] =(double)data.attribute(attribute1).addStringValue(articles.get(i).getTitle());
					}
					else if(attribute1.equals(Constants.DESCRIPTION_ATTRIBUTE))
					{
						newInst[0] =(double)data.attribute(attribute1).addStringValue(articles.get(i).getDescription());
					}
						
					if(attribute2.equals(Constants.SOURCE_ATTRIBUTE))
					{
						newInst[1] =(double)data.attribute(attribute2).addStringValue(articles.get(i).getSource());	
					}
					else if(attribute2.equals(Constants.TITLE_ATTRIBUTE))
					{
						newInst[1] =(double)data.attribute(attribute2).addStringValue(articles.get(i).getTitle());
					}
					else if(attribute2.equals(Constants.DESCRIPTION_ATTRIBUTE))
					{
						newInst[1] =(double)data.attribute(attribute2).addStringValue(articles.get(i).getDescription());
					}
					
					newInst[2] =(double)data.attribute(Constants.CATEGORY_ATTRIBUTE).addStringValue(articles.get(i).getCategory());
					
					//Assign Equal Weight for all instances
					data.add(new Instance(1.0, newInst));
				}
				
				//System.out.println("Arff Dataset is: " + data);
			}
			
	    } 
		catch (Exception e) {
			System.err.println("failed to create arff file. "+ e.getMessage());
		}
		return data;		
	}
	
	/*
	 * create_ArffFile_With_Weight() create an Arff file with attributes like source, title, description and category as Class attribute/ target attribute
	 * Here each attributes is assigned the user defined weights and instances will have equal weights of 1.0
	 * The combinations could lead to the following ARFF files
	 * 1. Arff File containing the XML tags like Source and Category
	 * 2. Arff File containing the XML tags like Title and Category
	 * 3. Arff File containing the XML tags like Description and Category
	 */
	public Instances create_ArffFile_With_Weight(String attribute1, double attribute1_weight) throws Exception
	{
		FastVector atts = new FastVector(2);
		atts.addElement(new Attribute(attribute1, (FastVector) null));
		atts.addElement(new Attribute(Constants.CATEGORY_ATTRIBUTE, (FastVector) null));
		
		Instances data = new Instances(Constants.RELATION_NAME, atts, 0);
		
		data.attribute(attribute1).setWeight(attribute1_weight);
				
		/*Class Attribute is the attribute to which the instance has to be classified. 
		Here we want all the class instances to be classified to category attribute*/
		data.setClass(data.attribute(Constants.CATEGORY_ATTRIBUTE)); 
		
		try {
			if(articles == null)
			{
				articles = readXMLFile();
			}			
			System.out.println("number of articles " + articles.size());
			if(articles.size() > 0)
			{
				for(int i=0;i<articles.size();i++)
				{
					double[] newInst = new double[2];
					
					//Check which attribute entered to add the value
					if(attribute1.equals(Constants.SOURCE_ATTRIBUTE))
					{
						newInst[0] =(double)data.attribute(attribute1).addStringValue(articles.get(i).getSource());	
					}
					else if(attribute1.equals(Constants.TITLE_ATTRIBUTE))
					{
						newInst[0] =(double)data.attribute(attribute1).addStringValue(articles.get(i).getTitle());
					}
					else if(attribute1.equals(Constants.DESCRIPTION_ATTRIBUTE))
					{
						newInst[0] =(double)data.attribute(attribute1).addStringValue(articles.get(i).getDescription());
					}
					
					newInst[1] =(double)data.attribute(Constants.CATEGORY_ATTRIBUTE).addStringValue(articles.get(i).getCategory());
					
					//Assign Equal Weight for all instances
					data.add(new Instance(1.0, newInst));
				}
				
				//System.out.println("Arff Dataset is: " + data);
			}
			
	    } 
		catch (Exception e) {
			System.err.println("failed to create arff file. "+ e.getMessage());
		}
		return data;		
	}

	/*
	 * create_ArffFile() create an Arff file with attributes like source, title, description and category as Class attribute/ target attribute
	 * Here all attributes created will have the same weights and instances will have same weights of 1.0
	 * The combinations could lead to the following ARFF files
	 * 1. Arff File containing the XML tags like Source, Title and Category
	 * 2. Arff File containing the XML tags like Title, Description and Category
	 * 3. Arff File containing the XML tags like Source, Description and Category
	 */
	public Instances create_ArffFile(String attribute1, String attribute2) throws Exception
	{
		FastVector atts = new FastVector(3);
		atts.addElement(new Attribute(attribute1, (FastVector) null));
		atts.addElement(new Attribute(attribute2, (FastVector) null));
		atts.addElement(new Attribute(Constants.CATEGORY_ATTRIBUTE, (FastVector) null));
		
		Instances data = new Instances(Constants.RELATION_NAME, atts, 0);
		
		/*Class Attribute is the attribute to which the instance has to be classified. 
		Here we want all the class instances to be classified to category attribute*/
		data.setClass(data.attribute(Constants.CATEGORY_ATTRIBUTE)); 
		
		try {
			if(articles == null)
			{
				articles = readXMLFile();
			}
			System.out.println("number of articles " + articles.size());
			if(articles.size() > 0)
			{
				for(int i=0;i<articles.size();i++)
				{
					double[] newInst = new double[3];
					
					//Check which attribute entered to add the value
					if(attribute1.equals(Constants.SOURCE_ATTRIBUTE))
					{
						newInst[0] =(double)data.attribute(attribute1).addStringValue(articles.get(i).getSource());	
					}
					else if(attribute1.equals(Constants.TITLE_ATTRIBUTE))
					{
						newInst[0] =(double)data.attribute(attribute1).addStringValue(articles.get(i).getTitle());
					}
					else if(attribute1.equals(Constants.DESCRIPTION_ATTRIBUTE))
					{
						newInst[0] =(double)data.attribute(attribute1).addStringValue(articles.get(i).getDescription());
					}
						
					if(attribute2.equals(Constants.SOURCE_ATTRIBUTE))
					{
						newInst[1] =(double)data.attribute(attribute2).addStringValue(articles.get(i).getSource());	
					}
					else if(attribute2.equals(Constants.TITLE_ATTRIBUTE))
					{
						newInst[1] =(double)data.attribute(attribute2).addStringValue(articles.get(i).getTitle());
					}
					else if(attribute2.equals(Constants.DESCRIPTION_ATTRIBUTE))
					{
						newInst[1] =(double)data.attribute(attribute2).addStringValue(articles.get(i).getDescription());
					}
					
					newInst[2] =(double)data.attribute(Constants.CATEGORY_ATTRIBUTE).addStringValue(articles.get(i).getCategory());
					
					//Assign Equal Weight for all instances
					data.add(new Instance(1.0, newInst));
				}
				
				//System.out.println("Arff Dataset is: " + data);
			}
			
	    } 
		catch (Exception e) {
			System.err.println("failed to create arff file. "+ e.getMessage());
		}
		return data;		
	}

	/*
	 * create_ArffFile_With_Weight() create an Arff file with attributes like source, title, description and category as Class attribute/ target attribute
	 * Here each attributes is assigned the user defined weights and instances will have equal weights of 1.0
	 * The combinations could lead to the following ARFF files
	 * 1. Arff File containing the XML tags like Source and Category
	 * 2. Arff File containing the XML tags like Title and Category
	 * 3. Arff File containing the XML tags like Description and Category
	 */
	public Instances create_ArffFile(String attribute1) throws Exception
	{
		FastVector atts = new FastVector(2);
		atts.addElement(new Attribute(attribute1, (FastVector) null));
		atts.addElement(new Attribute(Constants.CATEGORY_ATTRIBUTE, (FastVector) null));
		
		Instances data = new Instances(Constants.RELATION_NAME, atts, 0);
		
		/*Class Attribute is the attribute to which the instance has to be classified. 
		Here we want all the class instances to be classified to category attribute*/
		data.setClass(data.attribute(Constants.CATEGORY_ATTRIBUTE)); 
		
		try {
			if(articles == null)
			{
				articles = readXMLFile();
			}
			System.out.println("number of articles " + articles.size());
			if(articles.size() > 0)
			{
				for(int i=0;i<articles.size();i++)
				{
					double[] newInst = new double[2];
					
					//Check which attribute entered to add the value
					if(attribute1.equals(Constants.SOURCE_ATTRIBUTE))
					{
						newInst[0] =(double)data.attribute(attribute1).addStringValue(articles.get(i).getSource());	
					}
					else if(attribute1.equals(Constants.TITLE_ATTRIBUTE))
					{
						newInst[0] =(double)data.attribute(attribute1).addStringValue(articles.get(i).getTitle());
					}
					else if(attribute1.equals(Constants.DESCRIPTION_ATTRIBUTE))
					{
						newInst[0] =(double)data.attribute(attribute1).addStringValue(articles.get(i).getDescription());
					}
					
					newInst[1] =(double)data.attribute(Constants.CATEGORY_ATTRIBUTE).addStringValue(articles.get(i).getCategory());
					
					//Assign Equal Weight for all instances
					data.add(new Instance(1.0, newInst));
				}				
				//System.out.println("Arff Dataset is: " + data);
			}
			
	    } 
		catch (Exception e) {
			System.err.println("failed to create arff file. "+ e.getMessage());
		}
		return data;		
	}

}
